package com.sxw.github.webmagicdemo;

import com.sxw.github.webmagicdemo.webmagic.FangPipeline;
import com.sxw.github.webmagicdemo.webmagic.FangProcess;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import us.codecraft.webmagic.Spider;

@RunWith(SpringRunner.class)
@SpringBootTest
public class Fang {
    @Autowired
    private FangProcess fangProcess;

    @Autowired
    private FangPipeline fangPipeline;

    @Test
    public void spiderFang(){
        Spider spider = Spider.create(fangProcess);
        for(int i=1;i<=100;i++){
            spider.addUrl("http://jn.esf.fang.com/house/i3"+i+"/");
        }
        spider.addPipeline(fangPipeline)
                .thread(10)
                .run();
    }
}
